<!DOCTYPE html>
<html lang="en">

<!-- Head tag -->
<head><meta name="generator" content="Hexo 3.9.0">

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">

    <!--Description-->
    
        <meta name="description" content="线程池(适当使用)
1234567891011121314151617181920212223242526272829303132333435363738import reimport requestsfrom lxml import etreefrom multiprocessing.dummy ">
    

    <!--Author-->
    
        <meta name="author" content="ck">
    

    <!--Open Graph Title-->
    
        <meta property="og:title" content="高性能异步爬虫">
    

    <!--Open Graph Description-->
    

    <!--Open Graph Site Name-->
    <meta property="og:site_name" content="CK&#39;s blogs">

    <!--Type page-->
    
        <meta property="og:type" content="article">
    

    <!--Page Cover-->
    

    <meta name="twitter:card" content="summary">
    

    <!-- Title -->
    
    <title>高性能异步爬虫 - CK&#39;s blogs</title>

    <!-- Bootstrap Core CSS -->
    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.2/css/bootstrap.min.css" integrity="sha384-y3tfxAZXuh4HwSYylfB+J125MxIs6mR5FOHamPBG064zB+AFeWH94NdvaCBm8qnd" crossorigin="anonymous">

    <!-- Custom Fonts -->
    <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">

    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
        <script src="//oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
        <script src="//oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
    <![endif]-->

    <!-- Gallery -->
    <link href="//cdnjs.cloudflare.com/ajax/libs/featherlight/1.3.5/featherlight.min.css" type="text/css" rel="stylesheet">

    <!-- Custom CSS -->
    <link rel="stylesheet" href="/blog/css/style.css">

    <!-- Google Analytics -->
    


</head>


<body>

<div class="bg-gradient"></div>
<div class="bg-pattern"></div>

<!-- Menu -->
<!--Menu Links and Overlay-->
<div class="menu-bg">
    <div class="menu-container">
        <ul>
            
            <li class="menu-item">
                <a href="/blog/">
                    Home
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/archives">
                    Archives
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/2019/09/04/本人简历/">
                    About
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/tags">
                    Tags
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/categories">
                    Categories
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/contact.html">
                    Contact
                </a>
            </li>
            
        </ul>
    </div>
</div>

<!--Hamburger Icon-->
<nav>
    <a href="#menu"></a>
</nav>

<div class="container">

    <!-- Main Content -->
    <div class="row">
    <div class="col-sm-12">

        <!--Title and Logo-->
        <header>
    <div class="logo">
        <a href="/blog/"><i class="logo-icon fa fa-cube" aria-hidden="true"></i></a>
        
    </div>
</header>

        <section class="main">
            
<div class="post">

    <div class="post-header">
        <h1 class="title">
            <a href="/blog/2019/08/01/高性能异步爬虫/">
                高性能异步爬虫
            </a>
        </h1>
        <div class="post-info">
            
                <span class="date">2019-08-01</span>
            
            
            
        </div>
    </div>

    <div class="content">

        <!-- Gallery -->
        

        <!-- Post Content -->
        <h1 id="线程池-适当使用"><a href="#线程池-适当使用" class="headerlink" title="线程池(适当使用)"></a>线程池(适当使用)</h1><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line">import re</span><br><span class="line">import requests</span><br><span class="line">from lxml import etree</span><br><span class="line">from multiprocessing.dummy import Pool</span><br><span class="line">import random</span><br><span class="line">headers = &#123;</span><br><span class="line">    &apos;User-Agent&apos;:&apos;Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36&apos;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">def request_video(url):</span><br><span class="line">    return requests.get(url=url,headers=headers).content</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">def saveVideo(data):</span><br><span class="line">    name = str(random.randint(0,9999))+&apos;.mp4&apos;</span><br><span class="line">    with open(name,&apos;wb&apos;) as fp:</span><br><span class="line">        fp.write(data)</span><br><span class="line">        print(name,&apos;下载存储成功！！！&apos;)</span><br><span class="line"></span><br><span class="line">url = &apos;https://www.pearvideo.com/category_1&apos;</span><br><span class="line">page_text = requests.get(url=url,headers=headers).text</span><br><span class="line"></span><br><span class="line">tree = etree.HTML(page_text)</span><br><span class="line">li_list = tree.xpath(&apos;//ul[@id=&quot;listvideoListUl&quot;]/li&apos;)</span><br><span class="line">#实例化一个线程池对象</span><br><span class="line">pool = Pool(4)</span><br><span class="line">video_url_list = [] #所有的视频连接</span><br><span class="line">for li in li_list:</span><br><span class="line">    detail_url = &apos;https://www.pearvideo.com/&apos;+li.xpath(&apos;./div/a/@href&apos;)[0]</span><br><span class="line">    detail_page_text = requests.get(url=detail_url,headers=headers).text</span><br><span class="line">    ex = &apos;srcUrl=&quot;(.*?)&quot;,vdoUrl=&apos;</span><br><span class="line">    video_url = re.findall(ex,detail_page_text,re.S)[0]</span><br><span class="line">    video_url_list.append(video_url)</span><br><span class="line">#异步的获取4个视频的二进制数据</span><br><span class="line">video_data_list = pool.map(request_video,video_url_list)</span><br><span class="line"></span><br><span class="line">#进行视频的持久化存储</span><br><span class="line">pool.map(saveVideo,video_data_list)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<p><a href="https://www.cnblogs.com/wanglan/p/10821140.html#_labelTop" target="_blank" rel="noopener">回到目录</a></p>
<h1 id="单线程-异步协程-推荐"><a href="#单线程-异步协程-推荐" class="headerlink" title="单线程+异步协程(推荐)"></a>单线程+异步协程(推荐)</h1><ul>
<li>event_loop：事件循环，相当于一个无限循环，我们可以把一些函数注册到这个事件循环上，当满足某些条件的时候，函数就会被循环执行。程序是按照设定的顺序从头执行到尾，运行的次数也是完全按照设定。当在编写异步程序时，必然其中有部分程序的运行耗时是比较久的，需要先让出当前程序的控制权，让其在背后运行，让另一部分的程序先运行起来。当背后运行的程序完成后，也需要及时通知主程序已经完成任务可以进行下一步操作，但这个过程所需的时间是不确定的，需要主程序不断的监听状态，一旦收到了任务完成的消息，就开始进行下一步。loop就是这个持续不断的监视器。</li>
<li>coroutine：中文翻译叫协程，在 Python 中常指代为协程对象类型，我们可以将协程对象注册到事件循环中，它会被事件循环调用。我们可以使用 async 关键字来定义一个方法，这个方法在调用时不会立即被执行，而是返回一个协程对象。</li>
<li>task：任务，它是对协程对象的进一步封装，包含了任务的各个状态。</li>
<li>future：代表将来执行或还没有执行的任务，实际上和 task 没有本质区别。</li>
<li>另外我们还需要了解 async/await 关键字，它是从 Python 3.5 才出现的，专门用于定义协程。其中，async 定义一个协程，await 用来挂起阻塞方法的执行。</li>
</ul>
<h3 id="基本使用"><a href="#基本使用" class="headerlink" title="基本使用"></a>基本使用</h3><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">#基本使用</span><br><span class="line">import asyncio</span><br><span class="line">async def hello(name):</span><br><span class="line">    print(&apos;hello to :&apos;,name)</span><br><span class="line">#获取了一个协程对象</span><br><span class="line">c = hello(&apos;bobo&apos;)</span><br><span class="line"></span><br><span class="line">#创建一个事件循环对象</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line"></span><br><span class="line">#将协程对象注册到事件循环中，然后启动事件循环对象</span><br><span class="line">loop.run_until_complete(c)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<h3 id="task的使用"><a href="#task的使用" class="headerlink" title="task的使用"></a>task的使用</h3><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">#task的使用</span><br><span class="line">import asyncio</span><br><span class="line">async def hello(name):</span><br><span class="line">    print(&apos;hello to :&apos;,name)</span><br><span class="line"></span><br><span class="line">#获取了一个协程对象</span><br><span class="line">c = hello(&apos;bobo&apos;)</span><br><span class="line">#创建一个事件循环对象</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">#就协程进行进一步的封装，封装到了task对象中</span><br><span class="line">task = loop.create_task(c)</span><br><span class="line">print(task)</span><br><span class="line">loop.run_until_complete(task)</span><br><span class="line">print(task)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<h3 id="future"><a href="#future" class="headerlink" title="future"></a>future</h3><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">#future</span><br><span class="line">import asyncio</span><br><span class="line">async def hello(name):</span><br><span class="line">    print(&apos;hello to :&apos;,name)</span><br><span class="line"></span><br><span class="line">c = hello(&apos;bobo&apos;)</span><br><span class="line">task = asyncio.ensure_future(c)</span><br><span class="line">loop.run_until_complete(task)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<h3 id="绑定回调-task"><a href="#绑定回调-task" class="headerlink" title="绑定回调(task)"></a>绑定回调(task)</h3><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">import asyncio</span><br><span class="line">def callback(task):</span><br><span class="line">    print(&apos;i am callback:&apos;,task.result())</span><br><span class="line"></span><br><span class="line">async def hello(name):</span><br><span class="line">    print(&apos;hello to :&apos;,name)</span><br><span class="line">    return name</span><br><span class="line"></span><br><span class="line">c = hello(&apos;bobo&apos;)</span><br><span class="line"></span><br><span class="line">task = asyncio.ensure_future(c)</span><br><span class="line">#给任务对象绑定一个回调函数</span><br><span class="line">task.add_done_callback(callback)</span><br><span class="line">loop.run_until_complete(task)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<h3 id="多任务异步协程"><a href="#多任务异步协程" class="headerlink" title="多任务异步协程"></a>多任务异步协程</h3><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">import requests</span><br><span class="line">async def get_page(url):</span><br><span class="line">    print(&apos;正在下载：&apos;,url)</span><br><span class="line">    #之所以没有实现异步操作，原因是因为requests模块是一个非异步的模块</span><br><span class="line">    response = requests.get(url=url)</span><br><span class="line">    print(&apos;响应数据：&apos;,response.text)</span><br><span class="line">    print(&apos;下载成功：&apos;,url)</span><br><span class="line">start = time.time()</span><br><span class="line">urls = [</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;</span><br><span class="line">]</span><br><span class="line">tasks = []</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">for url in urls:</span><br><span class="line">    c = get_page(url)</span><br><span class="line">    task = asyncio.ensure_future(c)</span><br><span class="line">    tasks.append(task)</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line">print(&apos;总耗时：&apos;,time.time()-start)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<p>asyncio.sleep()</p>
<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">import asyncio</span><br><span class="line">async def request(url):</span><br><span class="line">    print(&apos;正在下载：&apos;,url)</span><br><span class="line">#     sleep(2) #非异步模块的代码：在此处如果存在非异步操作代码，则会彻底让asyncio失去异步的效果</span><br><span class="line">    await asyncio.sleep(2)</span><br><span class="line">    print(&apos;下载成功：&apos;,url)</span><br><span class="line">urls = [</span><br><span class="line">    &apos;www.baidu.com&apos;,</span><br><span class="line">    &apos;www.taobao.com&apos;,</span><br><span class="line">    &apos;www.sogou.com&apos;</span><br><span class="line">]</span><br><span class="line">start = time.time()</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">tasks = [] #任务列表，放置多个任务对象</span><br><span class="line">for url in urls:</span><br><span class="line">    c = request(url)</span><br><span class="line">    task = asyncio.ensure_future(c)</span><br><span class="line">    tasks.append(task)</span><br><span class="line">    </span><br><span class="line">#将多个任务对象对应的列表注册到事件循环中</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line">print(&apos;总耗时：&apos;,time.time()-start)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<h3 id="多任务异步操作应用到爬虫中"><a href="#多任务异步操作应用到爬虫中" class="headerlink" title="多任务异步操作应用到爬虫中"></a>多任务异步操作应用到爬虫中</h3><ul>
<li>环境安装：pip install aiohttp  支持异步的网络请求的模块</li>
</ul>
<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">import aiohttp</span><br><span class="line">import asyncio</span><br><span class="line"></span><br><span class="line">async def get_page(url):</span><br><span class="line">    async with aiohttp.ClientSession() as session:</span><br><span class="line">        async with await session.get(url=url) as response:</span><br><span class="line">            page_text = await response.text() #read()  json()</span><br><span class="line">            print(page_text)</span><br><span class="line">start = time.time()</span><br><span class="line">urls = [</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;</span><br><span class="line">]</span><br><span class="line">tasks = []</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">for url in urls:</span><br><span class="line">    c = get_page(url)</span><br><span class="line">    task = asyncio.ensure_future(c)</span><br><span class="line">    tasks.append(task)</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line">print(&apos;总耗时：&apos;,time.time()-start)</span><br></pre></td></tr></table></figure>

<p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<h3 id="实现数据解析—任务的绑定回调机制"><a href="#实现数据解析—任务的绑定回调机制" class="headerlink" title="实现数据解析—任务的绑定回调机制"></a>实现数据解析—任务的绑定回调机制</h3><p><a href="javascript:void(0);" target="_blank" rel="noopener"><img src="https://common.cnblogs.com/images/copycode.gif" alt="复制代码"></a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line">import aiohttp</span><br><span class="line">import asyncio</span><br><span class="line">#回调函数：解析响应数据</span><br><span class="line">def callback(task):</span><br><span class="line">    print(&apos;this is callback()&apos;)</span><br><span class="line">    #获取响应数据</span><br><span class="line">    page_text = task.result()</span><br><span class="line">    print(&apos;在回调函数中，实现数据解析&apos;)</span><br><span class="line">    </span><br><span class="line">async def get_page(url):</span><br><span class="line">    async with aiohttp.ClientSession() as session:</span><br><span class="line">        async with await session.get(url=url) as response:</span><br><span class="line">            page_text = await response.text() #read()  json()</span><br><span class="line">#             print(page_text)</span><br><span class="line">            return page_text</span><br><span class="line">start = time.time()</span><br><span class="line">urls = [</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/bobo&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/jay&apos;,</span><br><span class="line">    &apos;http://127.0.0.1:5000/tom&apos;</span><br><span class="line">]</span><br><span class="line">tasks = []</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">for url in urls:</span><br><span class="line">    c = get_page(url)</span><br><span class="line">    task = asyncio.ensure_future(c)</span><br><span class="line">    #给任务对象绑定回调函数用于解析响应数据</span><br><span class="line">    task.add_done_callback(callback)</span><br><span class="line">    tasks.append(task)</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line">print(&apos;总耗时：&apos;,time.time()-start)</span><br></pre></td></tr></table></figure>
    </div>

    

    
        <div class="post-tags">
            <i class="fa fa-tags" aria-hidden="true"></i>
            <a href="/blog/tags/爬虫/">#爬虫</a> <a href="/blog/tags/异步爬虫/">#异步爬虫</a>
        </div>
    

    <!-- Comments -->
    

</div>
        </section>

    </div>
</div>


</div>

<!-- Footer -->
<div class="push"></div>

<footer class="footer-content">
    <div class="container">
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-6 col-lg-6 footer-about">
                <h2>About</h2>
                <p>
                    This theme was developed by <a href="https://github.com/klugjo">Jonathan Klughertz</a>. The source code is available on Github. Create Websites. Make Magic.
                </p>
            </div>
            
    <div class="col-xs-6 col-sm-6 col-md-3 col-lg-3 recent-posts">
        <h2>Recent Posts</h2>
        <ul>
            
            <li>
                <a class="footer-post" href="/blog/2019/10/25/tars框架环境基础搭建/">tars框架环境基础搭建</a>
            </li>
            
            <li>
                <a class="footer-post" href="/blog/2019/10/20/认识tars框架/">认识tars框架</a>
            </li>
            
            <li>
                <a class="footer-post" href="/blog/2019/10/05/redis数据迁移备份与恢复/">redis数据迁移备份与恢复</a>
            </li>
            
            <li>
                <a class="footer-post" href="/blog/2019/09/21/负载均衡的5种策略/">负载均衡的5种策略</a>
            </li>
            
        </ul>
    </div>



            
        </div>
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
                <ul class="list-inline footer-social-icons">
                    
                    <li class="list-inline-item">
                        <a href="https://github.com/klugjo/hexo-theme-alpha-dust">
                            <span class="footer-icon-container">
                                <i class="fa fa-github"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://twitter.com/?lang=en">
                            <span class="footer-icon-container">
                                <i class="fa fa-twitter"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.facebook.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-facebook"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.instagram.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-instagram"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://dribbble.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-dribbble"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://plus.google.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-google-plus"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.behance.net/">
                            <span class="footer-icon-container">
                                <i class="fa fa-behance"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://500px.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-500px"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="mailto:test@example.com">
                            <span class="footer-icon-container">
                                <i class="fa fa-envelope-o"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="\#">
                            <span class="footer-icon-container">
                                <i class="fa fa-rss"></i>
                            </span>
                        </a>
                    </li>
                    
                </ul>
            </div>
        </div>
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
                <div class="footer-copyright">
                    @Untitled. All right reserved | Design & Hexo <a href="http://www.codeblocq.com/">Jonathan Klughertz</a>
                </div>
            </div>
        </div>
    </div>
</footer>

<!-- After footer scripts -->

<!-- jQuery -->
<script src="//code.jquery.com/jquery-2.1.4.min.js"></script>

<!-- Tween Max -->
<script src="//cdnjs.cloudflare.com/ajax/libs/gsap/1.18.5/TweenMax.min.js"></script>

<!-- Gallery -->
<script src="//cdnjs.cloudflare.com/ajax/libs/featherlight/1.3.5/featherlight.min.js" type="text/javascript" charset="utf-8"></script>

<!-- Custom JavaScript -->
<script src="/blog/js/main.js"></script>

<!-- Disqus Comments -->



</body>

</html>